{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib qt5\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "Alpha=0.1 #学习速率\n",
    "Beta=0.9 #对未来奖励的衰减度\n",
    "Epsilon_start=1#选择随机动作的概率 Epsilon\n",
    "Epsilon_stop=0.01\n",
    "decay_rate=0.00001\n",
    "N_Mesh=10\n",
    "N_States=100#假设网格为5*5\n",
    "Actions=['left','right','up','down']\n",
    "Obstacle=[3,5,16,21,23,27,29,31,33,34,56,57,66,67,76,78,81,86,96,98]\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Caculate_coordinate(State):\n",
    "    X_Axis=0.5+(State-1)%N_Mesh\n",
    "    Y_Axis=0.5+((State-1)//N_Mesh)\n",
    "    return X_Axis,Y_Axis\n",
    "def Initial_Q_Table (N_States,Actions):#以状态数和动作为变量，返回Q表\n",
    "    Q_Table=np.zeros([N_States,len(Actions)])#初始化Q_表\n",
    "    Q_Table=pd.DataFrame(Q_Table,index=np.arange(1,N_States+1),columns=Actions)#Q_表定义行列标签\n",
    "    print ('初始化Q表完成') \n",
    "    return Q_Table\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Select_Action(State,Q_Table,ep):#根据目前的状态选择动作,返回执行动作\n",
    "    Epsilon=Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-decay_rate*ep)\n",
    "    States_Actions=Q_Table.loc[[State],:]#选择目前状态下，取出Q表的目前的状态行,state选择为数字\n",
    "    if(np.random.rand()<Epsilon or np.all(States_Actions==[0])):#其实np.all(States_Action==[0])也行\n",
    "        Execute_Action=np.random.choice(Actions)\n",
    "    else:\n",
    "        #Q_T=Q_Table.T\n",
    "        #G=(Q_Table.T).loc[:,[State]]\n",
    "        Execute_Action=np.array(pd.DataFrame.idxmax((Q_Table.T).loc[:,[State]]))[0]#这里真心累，有毒\n",
    "    return Execute_Action\n",
    "#———————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Next_State(State,Execute_Action):#根据目前状态个选择的动作,返回即时奖励和新状态\n",
    "    if(Execute_Action=='left'):\n",
    "        #目标在右上角，如果Execute_Action为‘left’，不可能到达目标\n",
    "        if(State%N_Mesh==1):#判断是不是在网格最左边\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State-1) in Obstacle):\n",
    "            State=State-1\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State-1\n",
    "            R=1\n",
    "    elif(Execute_Action=='right'):\n",
    "        if(State==N_States-1):\n",
    "            State='End'\n",
    "            R=100\n",
    "        elif(State%N_Mesh==0):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State+1) in Obstacle):\n",
    "            State=State+1\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State+1\n",
    "            R=1\n",
    "    elif(Execute_Action=='up'):\n",
    "        if(State==N_States-N_Mesh):\n",
    "            State='End'\n",
    "            R=100\n",
    "        elif(N_States+1-N_Mesh<=State<=N_States-1):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State+10) in Obstacle):\n",
    "            State=State+10\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State+N_Mesh\n",
    "            R=1\n",
    "    else:\n",
    "        if(0<=State<=N_Mesh):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State-10) in Obstacle):\n",
    "            State=State-10\n",
    "            R=-100\n",
    "        else:\n",
    "            R=0\n",
    "            State=State-10\n",
    "    return State,R\n",
    "        \n",
    "#————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Run_Function():\n",
    "    #——————————————————————————————————————————————————画图程序部分\n",
    "    \n",
    "    \n",
    "    #——————————————————————————————————————————————————\n",
    "    Q_Table=Initial_Q_Table (N_States,Actions)\n",
    "    Step=0 \n",
    "    for episode in np.arange(1000) :#跑20回合\n",
    "        if(episode%300==0):\n",
    "            print('Q_Table,%d'%episode)\n",
    "            Q_Table\n",
    "#         fig=plt.figure()\n",
    "#         ax=fig.gca()\n",
    "#         ax.set(xlim=[0, N_Mesh], ylim=[0, N_Mesh])\n",
    "#         ax.set_xticks(np.arange(0,(N_Mesh+1)))\n",
    "#         ax.set_yticks(np.arange(0,(N_Mesh+1)))\n",
    "#         plt.grid()\n",
    "        print('episode is %d' % episode)\n",
    "        if(np.random.rand()<0.4):\n",
    "            State_=1 #定义起点 \n",
    "            #print(\"Normal_start:%d\"%State_)\n",
    "        else:\n",
    "            State_=np.random.randint(low=5, high=100)\n",
    "            while(State_ in Obstacle):\n",
    "                State_=np.random.randint(low=5, high=100)\n",
    "            #print(\"Random_start:%d\"%State_)\n",
    "        Is_End=False\n",
    "        while not Is_End:\n",
    "            A=Select_Action(State_,Q_Table,Step)\n",
    "            Next_S,R=Next_State(State_,A)\n",
    "            print(Next_S)\n",
    "            Q_Table_S_A=Q_Table.loc[[State_],[A]]\n",
    "            if Next_S!='End':\n",
    "                D_T=np.array((Q_Table.loc[[Next_S],:]).T)\n",
    "                Q_Target=R+Beta*D_T.max() #Next_S状态中的可选动作中的最大Q值动作\n",
    "                Step+=1\n",
    "            else:\n",
    "                Q_Target=R\n",
    "                Step+=1\n",
    "                Is_End=True #目的是结束循\n",
    "#————————————————————————————————————————————————————————————————————————————画图程序\n",
    "#             X_State,Y_State=Caculate_coordinate(State_)\n",
    "#             if(Next_S)=='End':\n",
    "#                 H=N_States\n",
    "#             else:\n",
    "#                 H=Next_S\n",
    "#             X_Next_State,Y_Next_State=Caculate_coordinate(H)                      \n",
    "#             if(Step==0):\n",
    "#                 plt.scatter(X_State,Y_State)  \n",
    "#             else:\n",
    "#                 plt.scatter(X_State,Y_State) \n",
    "#                 plt.scatter(X_Next_State,Y_Next_State)\n",
    "#                 plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "#————————————————————————————————————————————————————————————————————————————\n",
    "            Q_Table.loc[[State_],[A]]+=Alpha*(Q_Target-Q_Table_S_A)\n",
    "            State_=Next_S\n",
    "        print(Step)\n",
    "        print(Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-decay_rate*Step))\n",
    "        #plt.show()\n",
    "        #print(Q_Table)\n",
    "    return Q_Table\n",
    "#————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "Q__Table=Run_Function()\n",
    "Q__Table\n",
    "Final_Q_Table=np.array(Q__Table)\n",
    "#—————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Plot_Final_Graph(Q_TABLE):\n",
    "    f=open('A.txt','w+')\n",
    "    f.write(str(Q_TABLE))\n",
    "    f.close()\n",
    "    fig=plt.figure()\n",
    "    ax=fig.gca()\n",
    "    ax.set(xlim=[0, N_Mesh], ylim=[0, N_Mesh])\n",
    "    ax.set_xticks(np.arange(0,(N_Mesh+1)))\n",
    "    ax.set_yticks(np.arange(0,(N_Mesh+1)))\n",
    "    plt.grid()\n",
    "    State_Start=1\n",
    "    State=State_Start\n",
    "    Step=1\n",
    "    while(State!=N_States):\n",
    "        Action=np.argmax((Q_TABLE[State-1]))\n",
    "        if(Action==0):\n",
    "             Next_State=State-1\n",
    "        elif(Action==1):\n",
    "             Next_State=State+1\n",
    "        elif(Action==2):\n",
    "             Next_State=State+N_Mesh\n",
    "        else:\n",
    "             Next_State=State-N_Mesh\n",
    "        Step+=1\n",
    "        print(Next_State)\n",
    "        X_State,Y_State=Caculate_coordinate(State)\n",
    "        X_Next_State,Y_Next_State=Caculate_coordinate(Next_State)\n",
    "        if(Step==0):\n",
    "            plt.scatter(X_State,Y_State)  \n",
    "        else:\n",
    "            plt.scatter(X_State,Y_State) \n",
    "            plt.scatter(X_Next_State,Y_Next_State)\n",
    "            plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "        for J in np.arange(len(Obstacle)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle[J]))[0],(Caculate_coordinate(Obstacle[J]))[1],marker=\"x\")\n",
    "        #print(X_State,Y_State)\n",
    "        #print(X_Next_State,Y_Next_State)\n",
    "        State=Next_State\n",
    "    plt.show()\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "Plot_Final_Graph(Final_Q_Table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "初始化Q表完成\n",
      "Q_Table,0\n",
      "episode is 0\n",
      "Normal_start:1\n",
      "853\n",
      "0.991591214456\n",
      "episode is 1\n",
      "Random_start:24\n",
      "1808\n",
      "0.982261637992\n",
      "episode is 2\n",
      "Random_start:31\n",
      "1907\n",
      "0.98129957527\n",
      "episode is 3\n",
      "Random_start:81\n",
      "1941\n",
      "0.98096938955\n",
      "episode is 4\n",
      "Normal_start:1\n",
      "2483\n",
      "0.975720971519\n",
      "episode is 5\n",
      "Normal_start:1\n",
      "3306\n",
      "0.967805703927\n",
      "episode is 6\n",
      "Random_start:46\n",
      "3439\n",
      "0.966532669096\n",
      "episode is 7\n",
      "Random_start:17\n",
      "3991\n",
      "0.961267154952\n",
      "episode is 8\n",
      "Random_start:26\n",
      "4188\n",
      "0.959395003332\n",
      "episode is 9\n",
      "Normal_start:1\n",
      "4634\n",
      "0.955170130088\n",
      "episode is 10\n",
      "Random_start:84\n",
      "4689\n",
      "0.954650429447\n",
      "episode is 11\n",
      "Normal_start:1\n",
      "4827\n",
      "0.953347710937\n",
      "episode is 12\n",
      "Random_start:64\n",
      "4863\n",
      "0.953008166882\n",
      "episode is 13\n",
      "Normal_start:1\n",
      "4958\n",
      "0.952112734521\n",
      "episode is 14\n",
      "Random_start:69\n",
      "4973\n",
      "0.95197142821\n",
      "episode is 15\n",
      "Normal_start:1\n",
      "5710\n",
      "0.945054618536\n",
      "episode is 16\n",
      "Random_start:56\n",
      "5728\n",
      "0.944886323851\n",
      "episode is 17\n",
      "Normal_start:1\n",
      "6008\n",
      "0.942272303481\n",
      "episode is 18\n",
      "Normal_start:1\n",
      "6338\n",
      "0.939200875523\n",
      "episode is 19\n",
      "Random_start:14\n",
      "6456\n",
      "0.938105065145\n",
      "episode is 20\n",
      "Random_start:81\n",
      "6778\n",
      "0.935121373157\n",
      "episode is 21\n",
      "Normal_start:1\n",
      "6906\n",
      "0.933937975336\n",
      "episode is 22\n",
      "Random_start:46\n",
      "6947\n",
      "0.933559238412\n",
      "episode is 23\n",
      "Random_start:93\n",
      "7031\n",
      "0.932783774392\n",
      "episode is 24\n",
      "Random_start:44\n",
      "7285\n",
      "0.930442877803\n",
      "episode is 25\n",
      "Random_start:66\n",
      "7794\n",
      "0.925769726813\n",
      "episode is 26\n",
      "Random_start:70\n",
      "7898\n",
      "0.924817821374\n",
      "episode is 27\n",
      "Random_start:18\n",
      "8605\n",
      "0.918372869129\n",
      "episode is 28\n",
      "Normal_start:1\n",
      "8748\n",
      "0.917074824249\n",
      "episode is 29\n",
      "Normal_start:1\n",
      "9301\n",
      "0.912072544523\n",
      "episode is 30\n",
      "Normal_start:1\n",
      "9345\n",
      "0.911675719911\n",
      "episode is 31\n",
      "Normal_start:1\n",
      "9490\n",
      "0.910369237546\n",
      "episode is 32\n",
      "Normal_start:1\n",
      "9706\n",
      "0.908426538862\n",
      "episode is 33\n",
      "Random_start:17\n",
      "9829\n",
      "0.907322153556\n",
      "episode is 34\n",
      "Normal_start:1\n",
      "9907\n",
      "0.90662251517\n",
      "episode is 35\n",
      "Random_start:26\n",
      "9983\n",
      "0.905941340938\n",
      "episode is 36\n",
      "Normal_start:1\n",
      "10492\n",
      "0.901392585915\n",
      "episode is 37\n",
      "Normal_start:1\n",
      "10773\n",
      "0.898891288717\n",
      "episode is 38\n",
      "Random_start:54\n",
      "10840\n",
      "0.898295931021\n",
      "episode is 39\n",
      "Random_start:19\n",
      "10893\n",
      "0.897825258916\n",
      "episode is 40\n",
      "Normal_start:1\n",
      "11079\n",
      "0.896175438743\n",
      "episode is 41\n",
      "Normal_start:1\n",
      "11224\n",
      "0.894891415499\n",
      "episode is 42\n",
      "Random_start:23\n",
      "11385\n",
      "0.893467886568\n",
      "episode is 43\n",
      "Normal_start:1\n",
      "11490\n",
      "0.892540732128\n",
      "episode is 44\n",
      "Normal_start:1\n",
      "11604\n",
      "0.891535208951\n",
      "episode is 45\n",
      "Random_start:31\n",
      "11648\n",
      "0.891147418779\n",
      "episode is 46\n",
      "Normal_start:1\n",
      "11750\n",
      "0.890249106629\n",
      "episode is 47\n",
      "Random_start:79\n",
      "12213\n",
      "0.886182973627\n",
      "episode is 48\n",
      "Normal_start:1\n",
      "12302\n",
      "0.88540351769\n",
      "episode is 49\n",
      "Random_start:38\n",
      "12577\n",
      "0.882999465103\n",
      "episode is 50\n",
      "Random_start:83\n",
      "12616\n",
      "0.882659061695\n",
      "episode is 51\n",
      "Random_start:87\n",
      "12651\n",
      "0.882353684468\n",
      "episode is 52\n",
      "Normal_start:1\n",
      "12732\n",
      "0.881647364082\n",
      "episode is 53\n",
      "Normal_start:1\n",
      "13229\n",
      "0.877326024107\n",
      "episode is 54\n",
      "Random_start:88\n",
      "13283\n",
      "0.876857794487\n",
      "episode is 55\n",
      "Normal_start:1\n",
      "13364\n",
      "0.87615592397\n",
      "episode is 56\n",
      "Random_start:23\n",
      "13481\n",
      "0.875143114148\n",
      "episode is 57\n",
      "Random_start:50\n",
      "13507\n",
      "0.874918206178\n",
      "episode is 58\n",
      "Random_start:56\n",
      "13522\n",
      "0.874788478177\n",
      "episode is 59\n",
      "Normal_start:1\n",
      "13724\n",
      "0.873043368605\n",
      "episode is 60\n",
      "Normal_start:1\n",
      "14106\n",
      "0.869752831863\n",
      "episode is 61\n",
      "Normal_start:1\n",
      "14403\n",
      "0.867203154098\n",
      "episode is 62\n",
      "Normal_start:1\n",
      "14567\n",
      "0.865798493062\n",
      "episode is 63\n",
      "Normal_start:1\n",
      "14716\n",
      "0.864524302815\n",
      "episode is 64\n",
      "Random_start:86\n",
      "14795\n",
      "0.8638494952\n",
      "episode is 65\n",
      "Random_start:45\n",
      "14893\n",
      "0.863013132579\n",
      "episode is 66\n",
      "Normal_start:1\n",
      "14956\n",
      "0.862475903551\n",
      "episode is 67\n",
      "Normal_start:1\n",
      "15140\n",
      "0.860908790075\n",
      "episode is 68\n",
      "Normal_start:1\n",
      "15283\n",
      "0.859692860102\n",
      "episode is 69\n",
      "Normal_start:1\n",
      "15357\n",
      "0.859064319974\n",
      "episode is 70\n",
      "Normal_start:1\n",
      "15495\n",
      "0.85789341932\n",
      "episode is 71\n",
      "Random_start:47\n",
      "15603\n",
      "0.856978188741\n",
      "episode is 72\n",
      "Normal_start:1\n",
      "15670\n",
      "0.856410903416\n",
      "episode is 73\n",
      "Random_start:88\n",
      "15701\n",
      "0.856148556702\n",
      "episode is 74\n",
      "Random_start:20\n",
      "15817\n",
      "0.855167593445\n",
      "episode is 75\n",
      "Normal_start:1\n",
      "15868\n",
      "0.854736667867\n",
      "episode is 76\n",
      "Random_start:7\n",
      "15994\n",
      "0.853672969936\n",
      "episode is 77\n",
      "Random_start:67\n",
      "16014\n",
      "0.853504252215\n",
      "episode is 78\n",
      "Random_start:52\n",
      "16174\n",
      "0.852155724521\n",
      "episode is 79\n",
      "Random_start:42\n",
      "16325\n",
      "0.850885028993\n",
      "episode is 80\n",
      "Normal_start:1\n",
      "16382\n",
      "0.850405861103\n",
      "episode is 81\n",
      "Random_start:93\n",
      "16400\n",
      "0.850254601661\n",
      "episode is 82\n",
      "Random_start:12\n",
      "16511\n",
      "0.849322436501\n",
      "episode is 83\n",
      "Normal_start:1\n",
      "16663\n",
      "0.848047635492\n",
      "episode is 84\n",
      "Random_start:98\n",
      "16665\n",
      "0.848030874707\n",
      "episode is 85\n",
      "Random_start:6\n",
      "16732\n",
      "0.847469582075\n",
      "episode is 86\n",
      "Normal_start:1\n",
      "16877\n",
      "0.846256131145\n",
      "episode is 87\n",
      "Random_start:54\n",
      "16943\n",
      "0.845704384195\n",
      "episode is 88\n",
      "Random_start:77\n",
      "17032\n",
      "0.844960938176\n",
      "episode is 89\n",
      "Random_start:47\n",
      "17055\n",
      "0.844768919243\n",
      "episode is 90\n",
      "Normal_start:1\n",
      "17159\n",
      "0.843901210853\n",
      "episode is 91\n",
      "Normal_start:1\n",
      "17280\n",
      "0.8428928006\n",
      "episode is 92\n",
      "Normal_start:1\n",
      "17396\n",
      "0.841927205105\n",
      "episode is 93\n",
      "Normal_start:1\n",
      "17867\n",
      "0.838018041276\n",
      "episode is 94\n",
      "Normal_start:1\n",
      "18039\n",
      "0.836595074347\n",
      "episode is 95\n",
      "Normal_start:1\n",
      "18172\n",
      "0.835496433657\n",
      "episode is 96\n",
      "Random_start:54\n",
      "18248\n",
      "0.83486929471\n",
      "episode is 97\n",
      "Normal_start:1\n",
      "18716\n",
      "0.831017925644\n",
      "episode is 98\n",
      "Normal_start:1\n",
      "18849\n",
      "0.82992669763\n",
      "episode is 99\n",
      "Random_start:97\n",
      "18911\n",
      "0.829418500635\n",
      "episode is 100\n",
      "Normal_start:1\n",
      "19008\n",
      "0.82862405006\n",
      "episode is 101\n",
      "Random_start:18\n",
      "19059\n",
      "0.828206658239\n",
      "episode is 102\n",
      "Random_start:79\n",
      "19089\n",
      "0.827961233057\n",
      "episode is 103\n",
      "Normal_start:1\n",
      "19212\n",
      "0.826955759233\n",
      "episode is 104\n",
      "Random_start:6\n",
      "19248\n",
      "0.826661708092\n",
      "episode is 105\n",
      "Random_start:15\n",
      "19485\n",
      "0.824728511587\n",
      "episode is 106\n",
      "Random_start:12\n",
      "19555\n",
      "0.824158401191\n",
      "episode is 107\n",
      "Random_start:28\n",
      "19607\n",
      "0.823735148877\n",
      "episode is 108\n",
      "Normal_start:1\n",
      "19756\n",
      "0.822523586344\n",
      "episode is 109\n",
      "Normal_start:1\n",
      "19899\n",
      "0.821362507984\n",
      "episode is 110\n",
      "Random_start:74\n",
      "19977\n",
      "0.82072989198\n",
      "episode is 111\n",
      "Normal_start:1\n",
      "20125\n",
      "0.819530899214\n",
      "episode is 112\n",
      "Random_start:45\n",
      "20159\n",
      "0.819255705493\n",
      "episode is 113\n",
      "Random_start:83\n",
      "20400\n",
      "0.817307747475\n",
      "episode is 114\n",
      "Normal_start:1\n",
      "20693\n",
      "0.814945797721\n",
      "episode is 115\n",
      "Normal_start:1\n",
      "20752\n",
      "0.814471019774\n",
      "episode is 116\n",
      "Random_start:22\n",
      "20905\n",
      "0.813241120227\n",
      "episode is 117\n",
      "Normal_start:1\n",
      "20993\n",
      "0.812534578965\n",
      "episode is 118\n",
      "Random_start:8\n",
      "21063\n",
      "0.811973001335\n",
      "episode is 119\n",
      "Random_start:20\n",
      "21074\n",
      "0.811884789156\n",
      "episode is 120\n",
      "Normal_start:1\n",
      "21145\n",
      "0.811315653023\n",
      "episode is 121\n",
      "Random_start:99\n",
      "21154\n",
      "0.81124353786\n",
      "episode is 122\n",
      "Random_start:30\n",
      "21327\n",
      "0.809858584869\n",
      "episode is 123\n",
      "Normal_start:1\n",
      "21568\n",
      "0.807933246644\n",
      "episode is 124\n",
      "Normal_start:1\n",
      "21640\n",
      "0.807358941481\n",
      "episode is 125\n",
      "Normal_start:1\n",
      "21678\n",
      "0.807056002645\n",
      "episode is 126\n",
      "Random_start:76\n",
      "21849\n",
      "0.805694201552\n",
      "episode is 127\n",
      "Random_start:93\n",
      "21885\n",
      "0.805407803195\n",
      "episode is 128\n",
      "Normal_start:1\n",
      "22030\n",
      "0.804255297648\n",
      "episode is 129\n",
      "Normal_start:1\n",
      "22118\n",
      "0.803556660432\n",
      "episode is 130\n",
      "Random_start:85\n",
      "22159\n",
      "0.803231368891\n",
      "episode is 131\n",
      "Random_start:52\n",
      "22330\n",
      "0.801876102333\n",
      "episode is 132\n",
      "Random_start:64\n",
      "22413\n",
      "0.801219117854\n",
      "episode is 133\n",
      "Random_start:53\n",
      "22509\n",
      "0.800459911978\n",
      "episode is 134\n",
      "Random_start:93\n",
      "22685\n",
      "0.79906992608\n",
      "episode is 135\n",
      "Random_start:77\n",
      "22906\n",
      "0.797328007072\n",
      "episode is 136\n",
      "Random_start:15\n",
      "22973\n",
      "0.796800673984\n",
      "episode is 137\n",
      "Random_start:84\n",
      "23045\n",
      "0.796234381389\n",
      "episode is 138\n",
      "Random_start:58\n",
      "23132\n",
      "0.795550654941\n",
      "episode is 139\n",
      "Normal_start:1\n",
      "23173\n",
      "0.795228645189\n",
      "episode is 140\n",
      "Random_start:50\n",
      "23181\n",
      "0.79516582941\n",
      "episode is 141\n",
      "Random_start:56\n",
      "23190\n",
      "0.795095167665\n",
      "episode is 142\n",
      "Random_start:78\n",
      "23205\n",
      "0.794977412222\n",
      "episode is 143\n",
      "Random_start:89\n",
      "23207\n",
      "0.794961712831\n",
      "episode is 144\n",
      "Normal_start:1\n",
      "23377\n",
      "0.793628411546\n",
      "episode is 145\n",
      "Random_start:83\n",
      "23436\n",
      "0.793166207147\n",
      "episode is 146\n",
      "Random_start:37\n",
      "23476\n",
      "0.792853003309\n",
      "episode is 147\n",
      "Normal_start:1\n",
      "23653\n",
      "0.79146857907\n",
      "episode is 148\n",
      "Random_start:7\n",
      "23693\n",
      "0.791156054147\n",
      "episode is 149\n",
      "Random_start:80\n",
      "23706\n",
      "0.791054510461\n",
      "episode is 150\n",
      "Normal_start:1\n",
      "23825\n",
      "0.7901256084\n",
      "episode is 151\n",
      "Random_start:83\n",
      "23975\n",
      "0.78895629719\n",
      "episode is 152\n",
      "Random_start:25\n",
      "24142\n",
      "0.787656525785\n",
      "episode is 153\n",
      "Random_start:99\n",
      "24143\n",
      "0.787648749258\n",
      "episode is 154\n",
      "Random_start:43\n",
      "24225\n",
      "0.787011338658\n",
      "episode is 155\n",
      "Normal_start:1\n",
      "24380\n",
      "0.785807903986\n",
      "episode is 156\n",
      "Random_start:39\n",
      "24444\n",
      "0.785311545779\n",
      "episode is 157\n",
      "Normal_start:1\n",
      "24535\n",
      "0.784606333193\n",
      "episode is 158\n",
      "Normal_start:1\n",
      "24637\n",
      "0.783816637546\n",
      "episode is 159\n",
      "Random_start:62\n",
      "24719\n",
      "0.783182367989\n",
      "episode is 160\n",
      "Random_start:48\n",
      "24789\n",
      "0.782641329717\n",
      "episode is 161\n",
      "Random_start:75\n",
      "24812\n",
      "0.782463642646\n",
      "episode is 162\n",
      "Normal_start:1\n",
      "24940\n",
      "0.781475521716\n",
      "episode is 163\n",
      "Random_start:6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25085\n",
      "0.780357692831\n",
      "episode is 164\n",
      "Random_start:42\n",
      "25216\n",
      "0.77934918497\n",
      "episode is 165\n",
      "Random_start:41\n",
      "25343\n",
      "0.778372731685\n",
      "episode is 166\n",
      "Random_start:23\n",
      "25391\n",
      "0.778004001276\n",
      "episode is 167\n",
      "Normal_start:1\n",
      "25491\n",
      "0.777236381149\n",
      "episode is 168\n",
      "Random_start:40\n",
      "25574\n",
      "0.776599839154\n",
      "episode is 169\n",
      "Random_start:82\n",
      "25607\n",
      "0.776346902944\n",
      "episode is 170\n",
      "Random_start:62\n",
      "25703\n",
      "0.775611562936\n",
      "episode is 171\n",
      "Normal_start:1\n",
      "25824\n",
      "0.774685733185\n",
      "episode is 172\n",
      "Random_start:37\n",
      "25893\n",
      "0.774158282021\n",
      "episode is 173\n",
      "Normal_start:1\n",
      "25972\n",
      "0.773554835371\n",
      "episode is 174\n",
      "Normal_start:1\n",
      "26060\n",
      "0.772883202677\n",
      "episode is 175\n",
      "Normal_start:1\n",
      "26143\n",
      "0.772250272322\n",
      "episode is 176\n",
      "Random_start:52\n",
      "26225\n",
      "0.771625483297\n",
      "episode is 177\n",
      "Normal_start:1\n",
      "26294\n",
      "0.771100142977\n",
      "episode is 178\n",
      "Normal_start:1\n",
      "26435\n",
      "0.770027747991\n",
      "episode is 179\n",
      "Random_start:77\n",
      "26535\n",
      "0.76926810013\n",
      "episode is 180\n",
      "Random_start:75\n",
      "26549\n",
      "0.769161810037\n",
      "episode is 181\n",
      "Random_start:49\n",
      "26679\n",
      "0.768175540898\n",
      "episode is 182\n",
      "Random_start:88\n",
      "26685\n",
      "0.76813005173\n",
      "episode is 183\n",
      "Random_start:62\n",
      "26860\n",
      "0.766804484349\n",
      "episode is 184\n",
      "Random_start:17\n",
      "26906\n",
      "0.766456434344\n",
      "episode is 185\n",
      "Normal_start:1\n",
      "26997\n",
      "0.765768372105\n",
      "episode is 186\n",
      "Random_start:16\n",
      "27026\n",
      "0.765549231054\n",
      "episode is 187\n",
      "Random_start:33\n",
      "27059\n",
      "0.765299940943\n",
      "episode is 188\n",
      "Normal_start:1\n",
      "27162\n",
      "0.764522382515\n",
      "episode is 189\n",
      "Normal_start:1\n",
      "27254\n",
      "0.763828541139\n",
      "episode is 190\n",
      "Normal_start:1\n",
      "27355\n",
      "0.763067558673\n",
      "episode is 191\n",
      "Random_start:29\n",
      "27433\n",
      "0.762480395001\n",
      "episode is 192\n",
      "Normal_start:1\n",
      "27511\n",
      "0.761893689138\n",
      "episode is 193\n",
      "Normal_start:1\n",
      "27548\n",
      "0.761615539934\n",
      "episode is 194\n",
      "Random_start:29\n",
      "27592\n",
      "0.761284901842\n",
      "episode is 195\n",
      "Random_start:98\n",
      "27625\n",
      "0.761037018727\n",
      "episode is 196\n",
      "Normal_start:1\n",
      "27784\n",
      "0.759843818713\n",
      "episode is 197\n",
      "Normal_start:1\n",
      "27865\n",
      "0.759236691139\n",
      "episode is 198\n",
      "Normal_start:1\n",
      "28015\n",
      "0.758113678573\n",
      "episode is 199\n",
      "Random_start:55\n",
      "28039\n",
      "0.757934152834\n",
      "episode is 200\n",
      "Normal_start:1\n",
      "28224\n",
      "0.756551753764\n",
      "episode is 201\n",
      "Normal_start:1\n",
      "28285\n",
      "0.756096496062\n",
      "episode is 202\n",
      "Random_start:17\n",
      "28317\n",
      "0.75585778338\n",
      "episode is 203\n",
      "Normal_start:1\n",
      "28427\n",
      "0.755037790897\n",
      "episode is 204\n",
      "Random_start:65\n",
      "28521\n",
      "0.754337784428\n",
      "episode is 205\n",
      "Normal_start:1\n",
      "28638\n",
      "0.753467418483\n",
      "episode is 206\n",
      "Random_start:49\n",
      "28652\n",
      "0.75336334033\n",
      "episode is 207\n",
      "Normal_start:1\n",
      "28774\n",
      "0.752456990041\n",
      "episode is 208\n",
      "Random_start:73\n",
      "28795\n",
      "0.752301090443\n",
      "episode is 209\n",
      "Random_start:57\n",
      "28839\n",
      "0.751974549808\n",
      "episode is 210\n",
      "Random_start:23\n",
      "28864\n",
      "0.751789079355\n",
      "episode is 211\n",
      "Random_start:62\n",
      "28928\n",
      "0.75131448623\n",
      "episode is 212\n",
      "Normal_start:1\n",
      "29047\n",
      "0.750432846671\n",
      "episode is 213\n",
      "Normal_start:1\n",
      "29163\n",
      "0.74957444254\n",
      "episode is 214\n",
      "Normal_start:1\n",
      "29199\n",
      "0.749308243659\n",
      "episode is 215\n",
      "Random_start:41\n",
      "29294\n",
      "0.748606234335\n",
      "episode is 216\n",
      "Normal_start:1\n",
      "29356\n",
      "0.7481484404\n",
      "episode is 217\n",
      "Random_start:11\n",
      "29473\n",
      "0.747285311754\n",
      "episode is 218\n",
      "Normal_start:1\n",
      "29522\n",
      "0.746924130448\n",
      "episode is 219\n",
      "Random_start:73\n",
      "29538\n",
      "0.746806232019\n",
      "episode is 220\n",
      "Random_start:15\n",
      "29600\n",
      "0.74634955374\n",
      "episode is 221\n",
      "Normal_start:1\n",
      "29732\n",
      "0.745378213555\n",
      "episode is 222\n",
      "Normal_start:1\n",
      "29792\n",
      "0.744937118968\n",
      "episode is 223\n",
      "Normal_start:1\n",
      "29837\n",
      "0.744606471666\n",
      "episode is 224\n",
      "Random_start:48\n",
      "29917\n",
      "0.7440190215\n",
      "episode is 225\n",
      "Random_start:49\n",
      "29936\n",
      "0.743879571134\n",
      "episode is 226\n",
      "Random_start:53\n",
      "30018\n",
      "0.743278036549\n",
      "episode is 227\n",
      "Normal_start:1\n",
      "30057\n",
      "0.742992113873\n",
      "episode is 228\n",
      "Normal_start:1\n",
      "30117\n",
      "0.742552450517\n",
      "episode is 229\n",
      "Random_start:52\n",
      "30207\n",
      "0.741893449906\n",
      "episode is 230\n",
      "Normal_start:1\n",
      "30265\n",
      "0.741469074786\n",
      "episode is 231\n",
      "Random_start:67\n",
      "30279\n",
      "0.741366676283\n",
      "episode is 232\n",
      "Normal_start:1\n",
      "30324\n",
      "0.741037635319\n",
      "episode is 233\n",
      "Normal_start:1\n",
      "30382\n",
      "0.740613756427\n",
      "episode is 234\n",
      "Normal_start:1\n",
      "30486\n",
      "0.739854313099\n",
      "episode is 235\n",
      "Random_start:69\n",
      "30510\n",
      "0.739679169082\n",
      "episode is 236\n",
      "Normal_start:1\n",
      "30655\n",
      "0.738621900992\n",
      "episode is 237\n",
      "Random_start:39\n",
      "30714\n",
      "0.738192140862\n",
      "episode is 238\n",
      "Normal_start:1\n",
      "30826\n",
      "0.737377022216\n",
      "episode is 239\n",
      "Random_start:8\n",
      "30890\n",
      "0.736911649857\n",
      "episode is 240\n",
      "Normal_start:1\n",
      "30937\n",
      "0.736570081656\n",
      "episode is 241\n",
      "Random_start:83\n",
      "31017\n",
      "0.735989058031\n",
      "episode is 242\n",
      "Normal_start:1\n",
      "31088\n",
      "0.735473788742\n",
      "episode is 243\n",
      "Normal_start:1\n",
      "31244\n",
      "0.734342931929\n",
      "episode is 244\n",
      "Normal_start:1\n",
      "31339\n",
      "0.7336551329\n",
      "episode is 245\n",
      "Normal_start:1\n",
      "31420\n",
      "0.733069209574\n",
      "episode is 246\n",
      "Random_start:50\n",
      "31478\n",
      "0.732649951029\n",
      "episode is 247\n",
      "Random_start:29\n",
      "31492\n",
      "0.732548787117\n",
      "episode is 248\n",
      "Random_start:38\n",
      "31506\n",
      "0.732447637368\n",
      "episode is 249\n",
      "Random_start:71\n",
      "31567\n",
      "0.732007078693\n",
      "episode is 250\n",
      "Random_start:13\n",
      "31794\n",
      "0.730369981433\n",
      "episode is 251\n",
      "Normal_start:1\n",
      "31859\n",
      "0.72990189309\n",
      "episode is 252\n",
      "Normal_start:1\n",
      "31992\n",
      "0.728945060007\n",
      "episode is 253\n",
      "Normal_start:1\n",
      "32025\n",
      "0.72870784728\n",
      "episode is 254\n",
      "Random_start:33\n",
      "32068\n",
      "0.72839886934\n",
      "episode is 255\n",
      "Random_start:20\n",
      "32134\n",
      "0.727924882519\n",
      "episode is 256\n",
      "Random_start:98\n",
      "32138\n",
      "0.727896166098\n",
      "episode is 257\n",
      "Random_start:68\n",
      "32152\n",
      "0.72779566767\n",
      "episode is 258\n",
      "Random_start:29\n",
      "32188\n",
      "0.727537307737\n",
      "episode is 259\n",
      "Random_start:17\n",
      "32219\n",
      "0.727314905646\n",
      "episode is 260\n",
      "Random_start:63\n",
      "32338\n",
      "0.726461808602\n",
      "episode is 261\n",
      "Random_start:71\n",
      "32435\n",
      "0.725767177598\n",
      "episode is 262\n",
      "Normal_start:1\n",
      "32546\n",
      "0.724973116816\n",
      "episode is 263\n",
      "Normal_start:1\n",
      "32599\n",
      "0.724594281464\n",
      "episode is 264\n",
      "Normal_start:1\n",
      "32704\n",
      "0.723844351251\n",
      "episode is 265\n",
      "Random_start:95\n",
      "32772\n",
      "0.723359102096\n",
      "episode is 266\n",
      "Normal_start:1\n",
      "32830\n",
      "0.72294547378\n",
      "episode is 267\n",
      "Random_start:69\n",
      "32868\n",
      "0.722674605968\n",
      "episode is 268\n",
      "Random_start:48\n",
      "32883\n",
      "0.722567712795\n",
      "episode is 269\n",
      "Normal_start:1\n",
      "33067\n",
      "0.721257793698\n",
      "episode is 270\n",
      "Normal_start:1\n",
      "33150\n",
      "0.720667694655\n",
      "episode is 271\n",
      "Normal_start:1\n",
      "33214\n",
      "0.720213012844\n",
      "episode is 272\n",
      "Random_start:51\n",
      "33262\n",
      "0.719872192401\n",
      "episode is 273\n",
      "Normal_start:1\n",
      "33305\n",
      "0.719567012977\n",
      "episode is 274\n",
      "Random_start:15\n",
      "33346\n",
      "0.719276150132\n",
      "episode is 275\n",
      "Random_start:36\n",
      "33377\n",
      "0.719056308603\n",
      "episode is 276\n",
      "Random_start:85\n",
      "33422\n",
      "0.718737305045\n",
      "episode is 277\n",
      "Random_start:25\n",
      "33491\n",
      "0.718248444981\n",
      "episode is 278\n",
      "Random_start:27\n",
      "33538\n",
      "0.717915646425\n",
      "episode is 279\n",
      "Random_start:79\n",
      "33582\n",
      "0.717604232057\n",
      "episode is 280\n",
      "Random_start:14\n",
      "33645\n",
      "0.717158581786\n",
      "episode is 281\n",
      "Random_start:73\n",
      "33776\n",
      "0.716232810556\n",
      "episode is 282\n",
      "Normal_start:1\n",
      "33826\n",
      "0.715879782415\n",
      "episode is 283\n",
      "Random_start:55\n",
      "33871\n",
      "0.715562207973\n",
      "episode is 284\n",
      "Random_start:18\n",
      "33897\n",
      "0.715378785644\n",
      "episode is 285\n",
      "Random_start:22\n",
      "33986\n",
      "0.714751277808\n",
      "episode is 286\n",
      "Random_start:65\n",
      "34002\n",
      "0.714638526624\n",
      "episode is 287\n",
      "Normal_start:1\n",
      "34030\n",
      "0.714441255455\n",
      "episode is 288\n",
      "Random_start:93\n",
      "34060\n",
      "0.714229954775\n",
      "episode is 289\n",
      "Random_start:24\n",
      "34107\n",
      "0.713899044467\n",
      "episode is 290\n",
      "Random_start:56\n",
      "34125\n",
      "0.713772354041\n",
      "episode is 291\n",
      "Normal_start:1\n",
      "34222\n",
      "0.71309002584\n",
      "episode is 292\n",
      "Random_start:75\n",
      "34246\n",
      "0.712921304482\n",
      "episode is 293\n",
      "Random_start:63\n",
      "34275\n",
      "0.712717486858\n",
      "episode is 294\n",
      "Random_start:90\n",
      "34276\n",
      "0.712710459718\n",
      "episode is 295\n",
      "Normal_start:1\n",
      "34329\n",
      "0.712338121853\n",
      "episode is 296\n",
      "Random_start:29\n",
      "34363\n",
      "0.712099367482\n",
      "episode is 297\n",
      "Random_start:61\n",
      "34510\n",
      "0.711068039624\n",
      "episode is 298\n",
      "Random_start:30\n",
      "34532\n",
      "0.71091382162\n",
      "episode is 299\n",
      "Random_start:86\n",
      "34555\n",
      "0.710752629978\n",
      "Q_Table,300\n",
      "episode is 300\n",
      "Normal_start:1\n",
      "34663\n",
      "0.70999622567\n",
      "episode is 301\n",
      "Random_start:16\n",
      "34692\n",
      "0.709793256196\n",
      "episode is 302\n",
      "Random_start:86\n",
      "34701\n",
      "0.709730277637\n",
      "episode is 303\n",
      "Random_start:8\n",
      "34742\n",
      "0.709443447028\n",
      "episode is 304\n",
      "Normal_start:1\n",
      "34822\n",
      "0.708884116032\n",
      "episode is 305\n",
      "Normal_start:1\n",
      "34872\n",
      "0.70853476132\n",
      "episode is 306\n",
      "Normal_start:1\n",
      "34950\n",
      "0.707990116646\n",
      "episode is 307\n",
      "Random_start:28\n",
      "34974\n",
      "0.707822619118\n",
      "episode is 308\n",
      "Random_start:62\n",
      "35011\n",
      "0.707564472509\n",
      "episode is 309\n",
      "Normal_start:1\n",
      "35091\n",
      "0.707006644092\n",
      "episode is 310\n",
      "Normal_start:1\n",
      "35164\n",
      "0.706498014914\n",
      "episode is 311\n",
      "Random_start:33\n",
      "35204\n",
      "0.706219471421\n",
      "episode is 312\n",
      "Normal_start:1\n",
      "35318\n",
      "0.705426233455\n",
      "episode is 313\n",
      "Random_start:72\n",
      "35360\n",
      "0.705134215765\n",
      "episode is 314\n",
      "Random_start:15\n",
      "35416\n",
      "0.704745049581\n",
      "episode is 315\n",
      "Random_start:88\n",
      "35419\n",
      "0.704724207542\n",
      "episode is 316\n",
      "Random_start:58\n",
      "35439\n",
      "0.704585276594\n",
      "episode is 317\n",
      "Random_start:87\n",
      "35448\n",
      "0.704522766732\n",
      "episode is 318\n",
      "Random_start:94\n",
      "35503\n",
      "0.704140884238\n",
      "episode is 319\n",
      "Normal_start:1\n",
      "35545\n",
      "0.703849406281\n",
      "episode is 320\n",
      "Random_start:26\n",
      "35589\n",
      "0.703544179697\n",
      "episode is 321\n",
      "Normal_start:1\n",
      "35689\n",
      "0.702850982174\n",
      "episode is 322\n",
      "Normal_start:1\n",
      "35775\n",
      "0.702255386472\n",
      "episode is 323\n",
      "Random_start:75\n",
      "35818\n",
      "0.701957780646\n",
      "episode is 324\n",
      "Random_start:83\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35871\n",
      "0.70159114019\n",
      "episode is 325\n",
      "Normal_start:1\n",
      "35929\n",
      "0.701190133632\n",
      "episode is 326\n",
      "Normal_start:1\n",
      "35999\n",
      "0.70070646984\n",
      "episode is 327\n",
      "Normal_start:1\n",
      "36134\n",
      "0.699774645229\n",
      "episode is 328\n",
      "Random_start:83\n",
      "36180\n",
      "0.699457421859\n",
      "episode is 329\n",
      "Random_start:79\n",
      "36185\n",
      "0.69942294985\n",
      "episode is 330\n",
      "Random_start:33\n",
      "36215\n",
      "0.699216153986\n",
      "episode is 331\n",
      "Normal_start:1\n",
      "36244\n",
      "0.69901631028\n",
      "episode is 332\n",
      "Random_start:45\n",
      "36283\n",
      "0.698747646312\n",
      "episode is 333\n",
      "Random_start:79\n",
      "36305\n",
      "0.698596138496\n",
      "episode is 334\n",
      "Random_start:20\n",
      "36339\n",
      "0.698362055606\n",
      "episode is 335\n",
      "Random_start:85\n",
      "36350\n",
      "0.698286339944\n",
      "episode is 336\n",
      "Random_start:60\n",
      "36392\n",
      "0.69799732038\n",
      "episode is 337\n",
      "Random_start:44\n",
      "36448\n",
      "0.697612149738\n",
      "episode is 338\n",
      "Random_start:43\n",
      "36562\n",
      "0.696828718528\n",
      "episode is 339\n",
      "Random_start:56\n",
      "36583\n",
      "0.696684499641\n",
      "episode is 340\n",
      "Random_start:79\n",
      "36592\n",
      "0.696622700817\n",
      "episode is 341\n",
      "Random_start:93\n",
      "36673\n",
      "0.696066761615\n",
      "episode is 342\n",
      "Random_start:35\n",
      "36731\n",
      "0.695668958267\n",
      "episode is 343\n",
      "Normal_start:1\n",
      "36808\n",
      "0.695141196384\n",
      "episode is 344\n",
      "Random_start:59\n",
      "36845\n",
      "0.694887741033\n",
      "episode is 345\n",
      "Normal_start:1\n",
      "36939\n",
      "0.694244249045\n",
      "episode is 346\n",
      "Normal_start:1\n",
      "36970\n",
      "0.694032166202\n",
      "episode is 347\n",
      "Random_start:27\n",
      "37066\n",
      "0.693375810424\n",
      "episode is 348\n",
      "Random_start:18\n",
      "37100\n",
      "0.693143502143\n",
      "episode is 349\n",
      "Normal_start:1\n",
      "37223\n",
      "0.692303752188\n",
      "episode is 350\n",
      "Random_start:41\n",
      "37275\n",
      "0.691949046468\n",
      "episode is 351\n",
      "Random_start:26\n",
      "37331\n",
      "0.691567261912\n",
      "episode is 352\n",
      "Normal_start:1\n",
      "37374\n",
      "0.691274250991\n",
      "episode is 353\n",
      "Random_start:49\n",
      "37396\n",
      "0.691124387141\n",
      "episode is 354\n",
      "Normal_start:1\n",
      "37446\n",
      "0.690783910074\n",
      "episode is 355\n",
      "Random_start:92\n",
      "37497\n",
      "0.690436798801\n",
      "episode is 356\n",
      "Normal_start:1\n",
      "37591\n",
      "0.689797488733\n",
      "episode is 357\n",
      "Normal_start:1\n",
      "37669\n",
      "0.689267453432\n",
      "episode is 358\n",
      "Normal_start:1\n",
      "37797\n",
      "0.68839854731\n",
      "episode is 359\n",
      "Normal_start:1\n",
      "37872\n",
      "0.687889939152\n",
      "episode is 360\n",
      "Random_start:79\n",
      "37885\n",
      "0.687801819188\n",
      "episode is 361\n",
      "Random_start:19\n",
      "37916\n",
      "0.687591733189\n",
      "episode is 362\n",
      "Random_start:10\n",
      "37972\n",
      "0.687212388045\n",
      "episode is 363\n",
      "Normal_start:1\n",
      "38033\n",
      "0.686799414458\n",
      "episode is 364\n",
      "Normal_start:1\n",
      "38090\n",
      "0.686413748717\n",
      "episode is 365\n",
      "Random_start:64\n",
      "38142\n",
      "0.686062105003\n",
      "episode is 366\n",
      "Normal_start:1\n",
      "38190\n",
      "0.685737673062\n",
      "episode is 367\n",
      "Normal_start:1\n",
      "38241\n",
      "0.685393134714\n",
      "episode is 368\n",
      "Normal_start:1\n",
      "38270\n",
      "0.685197299102\n",
      "episode is 369\n",
      "Random_start:18\n",
      "38304\n",
      "0.684967771042\n",
      "episode is 370\n",
      "Normal_start:1\n",
      "38371\n",
      "0.684515694098\n",
      "episode is 371\n",
      "Normal_start:1\n",
      "38394\n",
      "0.684360573328\n",
      "episode is 372\n",
      "Normal_start:1\n",
      "38446\n",
      "0.684009996988\n",
      "episode is 373\n",
      "Random_start:12\n",
      "38509\n",
      "0.683585504419\n",
      "episode is 374\n",
      "Normal_start:1\n",
      "38568\n",
      "0.683188206186\n",
      "episode is 375\n",
      "Random_start:13\n",
      "38622\n",
      "0.682824782688\n",
      "episode is 376\n",
      "Random_start:45\n",
      "38638\n",
      "0.682717139334\n",
      "episode is 377\n",
      "Random_start:67\n",
      "38685\n",
      "0.682401036569\n",
      "episode is 378\n",
      "Random_start:10\n",
      "38739\n",
      "0.682038038027\n",
      "episode is 379\n",
      "Random_start:65\n",
      "38790\n",
      "0.681695386012\n",
      "episode is 380\n",
      "Normal_start:1\n",
      "38852\n",
      "0.681279063946\n",
      "episode is 381\n",
      "Random_start:51\n",
      "38879\n",
      "0.681097843064\n",
      "episode is 382\n",
      "Normal_start:1\n",
      "38930\n",
      "0.680755670426\n",
      "episode is 383\n",
      "Normal_start:1\n",
      "38987\n",
      "0.680373448637\n",
      "episode is 384\n",
      "Random_start:69\n",
      "38996\n",
      "0.680313117742\n",
      "episode is 385\n",
      "Random_start:7\n",
      "39037\n",
      "0.680038345695\n",
      "episode is 386\n",
      "Random_start:69\n",
      "39046\n",
      "0.679978044958\n",
      "episode is 387\n",
      "Random_start:77\n",
      "39061\n",
      "0.679877555788\n",
      "episode is 388\n",
      "Random_start:11\n",
      "39136\n",
      "0.679375335977\n",
      "episode is 389\n",
      "Random_start:98\n",
      "39140\n",
      "0.679348561499\n",
      "episode is 390\n",
      "Normal_start:1\n",
      "39259\n",
      "0.678552510455\n",
      "episode is 391\n",
      "Normal_start:1\n",
      "39421\n",
      "0.67747033219\n",
      "episode is 392\n",
      "Random_start:28\n",
      "39444\n",
      "0.677316831666\n",
      "episode is 393\n",
      "Random_start:43\n",
      "39504\n",
      "0.67691656166\n",
      "episode is 394\n",
      "Normal_start:1\n",
      "39586\n",
      "0.676369914236\n",
      "episode is 395\n",
      "Random_start:54\n",
      "39687\n",
      "0.67569722039\n",
      "episode is 396\n",
      "Random_start:97\n",
      "39705\n",
      "0.675577405674\n",
      "episode is 397\n",
      "Normal_start:1\n",
      "39745\n",
      "0.675311227951\n",
      "episode is 398\n",
      "Random_start:32\n",
      "39780\n",
      "0.675078409767\n",
      "episode is 399\n",
      "Random_start:96\n",
      "39788\n",
      "0.675025205622\n",
      "episode is 400\n",
      "Random_start:13\n",
      "39863\n",
      "0.674526623709\n",
      "episode is 401\n",
      "Random_start:43\n",
      "39889\n",
      "0.674353869246\n",
      "episode is 402\n",
      "Normal_start:1\n",
      "40039\n",
      "0.673358085467\n",
      "episode is 403\n",
      "Normal_start:1\n",
      "40118\n",
      "0.672834239526\n",
      "episode is 404\n",
      "Random_start:28\n",
      "40153\n",
      "0.672602288136\n",
      "episode is 405\n",
      "Random_start:20\n",
      "40173\n",
      "0.672469780929\n",
      "episode is 406\n",
      "Random_start:64\n",
      "40264\n",
      "0.671867207641\n",
      "episode is 407\n",
      "Normal_start:1\n",
      "40336\n",
      "0.671390834767\n",
      "episode is 408\n",
      "Normal_start:1\n",
      "40395\n",
      "0.671000729266\n",
      "episode is 409\n",
      "Random_start:18\n",
      "40419\n",
      "0.670842108127\n",
      "episode is 410\n",
      "Normal_start:1\n",
      "40496\n",
      "0.67033345556\n",
      "episode is 411\n",
      "Normal_start:1\n",
      "40566\n",
      "0.669871383885\n",
      "episode is 412\n",
      "Random_start:74\n",
      "40599\n",
      "0.669653662254\n",
      "episode is 413\n",
      "Normal_start:1\n",
      "40654\n",
      "0.669290952494\n",
      "episode is 414\n",
      "Normal_start:1\n",
      "40710\n",
      "0.668921852918\n",
      "episode is 415\n",
      "Random_start:53\n",
      "40750\n",
      "0.668658336884\n",
      "episode is 416\n",
      "Random_start:76\n",
      "40784\n",
      "0.668434431116\n",
      "episode is 417\n",
      "Random_start:55\n",
      "40807\n",
      "0.668283008611\n",
      "episode is 418\n",
      "Random_start:34\n",
      "40837\n",
      "0.668085553328\n",
      "episode is 419\n",
      "Random_start:90\n",
      "40838\n",
      "0.668078972505\n",
      "episode is 420\n",
      "Normal_start:1\n",
      "40919\n",
      "0.667546144362\n",
      "episode is 421\n",
      "Normal_start:1\n",
      "40955\n",
      "0.667309470354\n",
      "episode is 422\n",
      "Random_start:23\n",
      "40994\n",
      "0.667053169642\n",
      "episode is 423\n",
      "Random_start:61\n",
      "41054\n",
      "0.666659055987\n",
      "episode is 424\n",
      "Random_start:49\n",
      "41068\n",
      "0.666567130154\n",
      "episode is 425\n",
      "Random_start:6\n",
      "41091\n",
      "0.666416137079\n",
      "episode is 426\n",
      "Normal_start:1\n",
      "41180\n",
      "0.665832186613\n",
      "episode is 427\n",
      "Normal_start:1\n",
      "41289\n",
      "0.665117718985\n",
      "episode is 428\n",
      "Normal_start:1\n",
      "41341\n",
      "0.664777146328\n",
      "episode is 429\n",
      "Normal_start:1\n",
      "41394\n",
      "0.664430206388\n",
      "episode is 430\n",
      "Random_start:92\n",
      "41439\n",
      "0.664135779046\n",
      "episode is 431\n",
      "Random_start:77\n",
      "41459\n",
      "0.664004964972\n",
      "episode is 432\n",
      "Normal_start:1\n",
      "41564\n",
      "0.663318620153\n",
      "episode is 433\n",
      "Normal_start:1\n",
      "41640\n",
      "0.662822286632\n",
      "episode is 434\n",
      "Random_start:29\n",
      "41674\n",
      "0.662600364784\n",
      "episode is 435\n",
      "Normal_start:1\n",
      "41725\n",
      "0.662267623454\n",
      "episode is 436\n",
      "Random_start:9\n",
      "41755\n",
      "0.662071972516\n",
      "episode is 437\n",
      "Random_start:6\n",
      "41780\n",
      "0.661908974898\n",
      "episode is 438\n",
      "Random_start:17\n",
      "41791\n",
      "0.661837268855\n",
      "episode is 439\n",
      "Normal_start:1\n",
      "41953\n",
      "0.660782147359\n",
      "episode is 440\n",
      "Random_start:65\n",
      "42045\n",
      "0.66018370311\n",
      "episode is 441\n",
      "Random_start:57\n",
      "42060\n",
      "0.660086182868\n",
      "episode is 442\n",
      "Random_start:84\n",
      "42087\n",
      "0.659910683292\n",
      "episode is 443\n",
      "Random_start:31\n",
      "42135\n",
      "0.659598801022\n",
      "episode is 444\n",
      "Random_start:20\n",
      "42149\n",
      "0.659507863556\n",
      "episode is 445\n",
      "Normal_start:1\n",
      "42239\n",
      "0.65892356945\n",
      "episode is 446\n",
      "Normal_start:1\n",
      "42284\n",
      "0.658631619538\n",
      "episode is 447\n",
      "Random_start:56\n",
      "42324\n",
      "0.658372218774\n",
      "episode is 448\n",
      "Random_start:16\n",
      "42358\n",
      "0.658151809691\n",
      "episode is 449\n",
      "Random_start:34\n",
      "42396\n",
      "0.657905558794\n",
      "episode is 450\n",
      "Normal_start:1\n",
      "42478\n",
      "0.657374494002\n",
      "episode is 451\n",
      "Normal_start:1\n",
      "42530\n",
      "0.657037946775\n",
      "episode is 452\n",
      "Normal_start:1\n",
      "42600\n",
      "0.656585178699\n",
      "episode is 453\n",
      "Random_start:65\n",
      "42616\n",
      "0.656481733347\n",
      "episode is 454\n",
      "Random_start:7\n",
      "42658\n",
      "0.65621026803\n",
      "episode is 455\n",
      "Random_start:80\n",
      "42666\n",
      "0.656158573277\n",
      "episode is 456\n",
      "Random_start:70\n",
      "42669\n",
      "0.65613918881\n",
      "episode is 457\n",
      "Random_start:68\n",
      "42702\n",
      "0.655925998056\n",
      "episode is 458\n",
      "Normal_start:1\n",
      "42725\n",
      "0.65577745216\n",
      "episode is 459\n",
      "Random_start:51\n",
      "42768\n",
      "0.655499827549\n",
      "episode is 460\n",
      "Normal_start:1\n",
      "42794\n",
      "0.65533201941\n",
      "episode is 461\n",
      "Random_start:96\n",
      "42892\n",
      "0.654699903819\n",
      "episode is 462\n",
      "Normal_start:1\n",
      "42955\n",
      "0.654293870793\n",
      "episode is 463\n",
      "Random_start:9\n",
      "42977\n",
      "0.654152141732\n",
      "episode is 464\n",
      "Normal_start:1\n",
      "43030\n",
      "0.653810831552\n",
      "episode is 465\n",
      "Random_start:35\n",
      "43064\n",
      "0.653591973078\n",
      "episode is 466\n",
      "Normal_start:1\n",
      "43133\n",
      "0.653148047788\n",
      "episode is 467\n",
      "Normal_start:1\n",
      "43198\n",
      "0.652730137392\n",
      "episode is 468\n",
      "Random_start:16\n",
      "43219\n",
      "0.652595178235\n",
      "episode is 469\n",
      "Random_start:8\n",
      "43245\n",
      "0.652428125206\n",
      "episode is 470\n",
      "Random_start:85\n",
      "43251\n",
      "0.652389580675\n",
      "episode is 471\n",
      "Random_start:65\n",
      "43300\n",
      "0.652074886887\n",
      "episode is 472\n",
      "Normal_start:1\n",
      "43375\n",
      "0.65159351126\n",
      "episode is 473\n",
      "Normal_start:1\n",
      "43412\n",
      "0.651356165573\n",
      "episode is 474\n",
      "Random_start:48\n",
      "43426\n",
      "0.651266381994\n",
      "episode is 475\n",
      "Normal_start:1\n",
      "43496\n",
      "0.650817652601\n",
      "episode is 476\n",
      "Random_start:77\n",
      "43505\n",
      "0.650759981607\n",
      "episode is 477\n",
      "Random_start:91\n",
      "43571\n",
      "0.650337219546\n",
      "episode is 478\n",
      "Random_start:57\n",
      "43583\n",
      "0.65026038369\n",
      "episode is 479\n",
      "Normal_start:1\n",
      "43653\n",
      "0.649812358249\n",
      "episode is 480\n",
      "Random_start:50\n",
      "43666\n",
      "0.649729188048\n",
      "episode is 481\n",
      "Normal_start:1\n",
      "43725\n",
      "0.64935185915\n",
      "episode is 482\n",
      "Random_start:69\n",
      "43740\n",
      "0.649255963564\n",
      "episode is 483\n",
      "Normal_start:1\n",
      "43806\n",
      "0.648834193827\n",
      "episode is 484\n",
      "Random_start:93\n",
      "43832\n",
      "0.648668118527\n",
      "episode is 485\n",
      "Normal_start:1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43866\n",
      "0.648451008278\n",
      "episode is 486\n",
      "Random_start:15\n",
      "43892\n",
      "0.648285032594\n",
      "episode is 487\n",
      "Random_start:18\n",
      "43921\n",
      "0.648099956771\n",
      "episode is 488\n",
      "Random_start:30\n",
      "43937\n",
      "0.647997868946\n",
      "episode is 489\n",
      "Random_start:63\n",
      "44002\n",
      "0.647583305079\n",
      "episode is 490\n",
      "Normal_start:1\n",
      "44057\n",
      "0.647232730678\n",
      "episode is 491\n",
      "Normal_start:1\n",
      "44138\n",
      "0.646716781154\n",
      "episode is 492\n",
      "Random_start:87\n",
      "44156\n",
      "0.646602182447\n",
      "episode is 493\n",
      "Normal_start:1\n",
      "44220\n",
      "0.646194887399\n",
      "episode is 494\n",
      "Random_start:67\n",
      "44230\n",
      "0.646131271091\n",
      "episode is 495\n",
      "Random_start:33\n",
      "44253\n",
      "0.645984977723\n",
      "episode is 496\n",
      "Random_start:76\n",
      "44283\n",
      "0.645794210846\n",
      "episode is 497\n",
      "Random_start:72\n",
      "44376\n",
      "0.645203197094\n",
      "episode is 498\n",
      "Normal_start:1\n",
      "44434\n",
      "0.64483488606\n",
      "episode is 499\n",
      "Normal_start:1\n",
      "44570\n",
      "0.643972097444\n",
      "episode is 500\n",
      "Normal_start:1\n",
      "44615\n",
      "0.643686874181\n",
      "episode is 501\n",
      "Normal_start:1\n",
      "44672\n",
      "0.643325775585\n",
      "episode is 502\n",
      "Normal_start:1\n",
      "44737\n",
      "0.642914247592\n",
      "episode is 503\n",
      "Random_start:46\n",
      "44755\n",
      "0.64280033328\n",
      "episode is 504\n",
      "Normal_start:1\n",
      "44819\n",
      "0.642395470637\n",
      "episode is 505\n",
      "Random_start:12\n",
      "44883\n",
      "0.641990867022\n",
      "episode is 506\n",
      "Normal_start:1\n",
      "44951\n",
      "0.641561259316\n",
      "episode is 507\n",
      "Random_start:42\n",
      "44978\n",
      "0.641390760794\n",
      "episode is 508\n",
      "Random_start:37\n",
      "44998\n",
      "0.641264495269\n",
      "episode is 509\n",
      "Random_start:67\n",
      "45024\n",
      "0.641100387835\n",
      "episode is 510\n",
      "Random_start:47\n",
      "45046\n",
      "0.640961561022\n",
      "episode is 511\n",
      "Random_start:13\n",
      "45066\n",
      "0.640835381328\n",
      "episode is 512\n",
      "Normal_start:1\n",
      "45135\n",
      "0.64040025505\n",
      "episode is 513\n",
      "Random_start:73\n",
      "45154\n",
      "0.64028049038\n",
      "episode is 514\n",
      "Random_start:34\n",
      "45187\n",
      "0.640072532133\n",
      "episode is 515\n",
      "Normal_start:1\n",
      "45240\n",
      "0.639738682169\n",
      "episode is 516\n",
      "Normal_start:1\n",
      "45289\n",
      "0.639430185803\n",
      "episode is 517\n",
      "Random_start:59\n",
      "45309\n",
      "0.639304312353\n",
      "episode is 518\n",
      "Normal_start:1\n",
      "45408\n",
      "0.638681609373\n",
      "episode is 519\n",
      "Random_start:39\n",
      "45426\n",
      "0.638568456867\n",
      "episode is 520\n",
      "Normal_start:1\n",
      "45502\n",
      "0.638090926325\n",
      "episode is 521\n",
      "Random_start:38\n",
      "45517\n",
      "0.637996719751\n",
      "episode is 522\n",
      "Random_start:36\n",
      "45537\n",
      "0.637871132966\n",
      "episode is 523\n",
      "Random_start:29\n",
      "45558\n",
      "0.637739293872\n",
      "episode is 524\n",
      "Random_start:55\n",
      "45609\n",
      "0.637419228456\n",
      "episode is 525\n",
      "Normal_start:1\n",
      "45670\n",
      "0.637036619434\n",
      "episode is 526\n",
      "Normal_start:1\n",
      "45700\n",
      "0.636848536662\n",
      "episode is 527\n",
      "Normal_start:1\n",
      "45722\n",
      "0.636710645153\n",
      "episode is 528\n",
      "Random_start:50\n",
      "45738\n",
      "0.636610379471\n",
      "episode is 529\n",
      "Normal_start:1\n",
      "45799\n",
      "0.636228263697\n",
      "episode is 530\n",
      "Normal_start:1\n",
      "45902\n",
      "0.635583580654\n",
      "episode is 531\n",
      "Random_start:50\n",
      "45914\n",
      "0.635508515128\n",
      "episode is 532\n",
      "Random_start:5\n",
      "45973\n",
      "0.635139573953\n",
      "episode is 533\n",
      "Normal_start:1\n",
      "46011\n",
      "0.634902066044\n",
      "episode is 534\n",
      "Random_start:75\n",
      "46056\n",
      "0.634620923376\n",
      "episode is 535\n",
      "Random_start:88\n",
      "46067\n",
      "0.634552218853\n",
      "episode is 536\n",
      "Random_start:60\n",
      "46087\n",
      "0.6344273209\n",
      "episode is 537\n",
      "Random_start:48\n",
      "46108\n",
      "0.63429620493\n",
      "episode is 538\n",
      "Random_start:26\n",
      "46145\n",
      "0.634065258062\n",
      "episode is 539\n",
      "Normal_start:1\n",
      "46210\n",
      "0.633659747449\n",
      "episode is 540\n",
      "Random_start:95\n",
      "46248\n",
      "0.633422801768\n",
      "episode is 541\n",
      "Normal_start:1\n",
      "46347\n",
      "0.632805918602\n",
      "episode is 542\n",
      "Random_start:97\n",
      "46366\n",
      "0.632687596718\n",
      "episode is 543\n",
      "Normal_start:1\n",
      "46416\n",
      "0.632376330743\n",
      "episode is 544\n",
      "Random_start:96\n",
      "46432\n",
      "0.632276758496\n",
      "episode is 545\n",
      "Normal_start:1\n",
      "46531\n",
      "0.631661009351\n",
      "episode is 546\n",
      "Random_start:98\n",
      "46551\n",
      "0.631536689581\n",
      "episode is 547\n",
      "Random_start:86\n",
      "46560\n",
      "0.631480753797\n",
      "episode is 548\n",
      "Normal_start:1\n",
      "46649\n",
      "0.63092788199\n",
      "episode is 549\n",
      "Random_start:80\n",
      "46653\n",
      "0.630903045372\n",
      "episode is 550\n",
      "Normal_start:1\n",
      "46748\n",
      "0.630313467572\n",
      "episode is 551\n",
      "Random_start:60\n",
      "46754\n",
      "0.630276249881\n",
      "episode is 552\n",
      "Random_start:12\n",
      "46782\n",
      "0.630102596843\n",
      "episode is 553\n",
      "Random_start:34\n",
      "46813\n",
      "0.629910394831\n",
      "episode is 554\n",
      "Random_start:41\n",
      "46889\n",
      "0.629439441916\n",
      "episode is 555\n",
      "Normal_start:1\n",
      "46942\n",
      "0.629111225997\n",
      "episode is 556\n",
      "Random_start:56\n",
      "46961\n",
      "0.628993606038\n",
      "episode is 557\n",
      "Normal_start:1\n",
      "47008\n",
      "0.6287027474\n",
      "episode is 558\n",
      "Random_start:69\n",
      "47021\n",
      "0.628622321271\n",
      "episode is 559\n",
      "Normal_start:1\n",
      "47072\n",
      "0.628306904325\n",
      "episode is 560\n",
      "Normal_start:1\n",
      "47215\n",
      "0.627423357339\n",
      "episode is 561\n",
      "Random_start:28\n",
      "47239\n",
      "0.627275193513\n",
      "episode is 562\n",
      "Normal_start:1\n",
      "47294\n",
      "0.626935785503\n",
      "episode is 563\n",
      "Normal_start:1\n",
      "47346\n",
      "0.626615062289\n",
      "episode is 564\n",
      "Normal_start:1\n",
      "47398\n",
      "0.626294505809\n",
      "episode is 565\n",
      "Random_start:55\n",
      "47428\n",
      "0.626109645188\n",
      "episode is 566\n",
      "Random_start:47\n",
      "47443\n",
      "0.626017235672\n",
      "episode is 567\n",
      "Random_start:78\n",
      "47449\n",
      "0.625980275746\n",
      "episode is 568\n",
      "Random_start:53\n",
      "47473\n",
      "0.625832458219\n",
      "episode is 569\n",
      "Random_start:49\n",
      "47490\n",
      "0.625727775599\n",
      "episode is 570\n",
      "Normal_start:1\n",
      "47525\n",
      "0.625512308587\n",
      "episode is 571\n",
      "Random_start:20\n",
      "47592\n",
      "0.625100053461\n",
      "episode is 572\n",
      "Normal_start:1\n",
      "47631\n",
      "0.624860211212\n",
      "episode is 573\n",
      "Random_start:59\n",
      "47657\n",
      "0.624700368338\n",
      "episode is 574\n",
      "Random_start:37\n",
      "47684\n",
      "0.624534421642\n",
      "episode is 575\n",
      "Normal_start:1\n",
      "47728\n",
      "0.624264085975\n",
      "episode is 576\n",
      "Random_start:15\n",
      "47758\n",
      "0.624079834388\n",
      "episode is 577\n",
      "Random_start:82\n",
      "47827\n",
      "0.623656265451\n",
      "episode is 578\n",
      "Random_start:85\n",
      "47862\n",
      "0.62344152334\n",
      "episode is 579\n",
      "Random_start:80\n",
      "47873\n",
      "0.623374048483\n",
      "episode is 580\n",
      "Normal_start:1\n",
      "47905\n",
      "0.623177800189\n",
      "episode is 581\n",
      "Normal_start:1\n",
      "48001\n",
      "0.622589431963\n",
      "episode is 582\n",
      "Random_start:95\n",
      "48039\n",
      "0.622356692202\n",
      "episode is 583\n",
      "Normal_start:1\n",
      "48099\n",
      "0.621989388389\n",
      "episode is 584\n",
      "Random_start:66\n",
      "48162\n",
      "0.621603956498\n",
      "episode is 585\n",
      "Normal_start:1\n",
      "48237\n",
      "0.621145425502\n",
      "episode is 586\n",
      "Normal_start:1\n",
      "48297\n",
      "0.62077884823\n",
      "episode is 587\n",
      "Random_start:50\n",
      "48303\n",
      "0.620742202599\n",
      "episode is 588\n",
      "Normal_start:1\n",
      "48378\n",
      "0.620284317675\n",
      "episode is 589\n",
      "Normal_start:1\n",
      "48445\n",
      "0.61987556413\n",
      "episode is 590\n",
      "Random_start:15\n",
      "48489\n",
      "0.619607277909\n",
      "episode is 591\n",
      "Random_start:47\n",
      "48536\n",
      "0.619320829809\n",
      "episode is 592\n",
      "Random_start:89\n",
      "48539\n",
      "0.619302550458\n",
      "episode is 593\n",
      "Random_start:30\n",
      "48575\n",
      "0.619083241018\n",
      "episode is 594\n",
      "Random_start:12\n",
      "48609\n",
      "0.618876187917\n",
      "episode is 595\n",
      "Random_start:81\n",
      "48638\n",
      "0.618699639424\n",
      "episode is 596\n",
      "Random_start:56\n",
      "48657\n",
      "0.618583997479\n",
      "episode is 597\n",
      "Normal_start:1\n",
      "48696\n",
      "0.618346695996\n",
      "episode is 598\n",
      "Random_start:50\n",
      "48705\n",
      "0.618291947257\n",
      "episode is 599\n",
      "Normal_start:1\n",
      "48740\n",
      "0.618079082329\n",
      "Q_Table,600\n",
      "episode is 600\n",
      "Normal_start:1\n",
      "48763\n",
      "0.617939240223\n",
      "episode is 601\n",
      "Random_start:12\n",
      "48815\n",
      "0.617623193997\n",
      "episode is 602\n",
      "Random_start:90\n",
      "48849\n",
      "0.617416637228\n",
      "episode is 603\n",
      "Random_start:11\n",
      "48909\n",
      "0.617052296559\n",
      "episode is 604\n",
      "Random_start:22\n",
      "48944\n",
      "0.616839865432\n",
      "episode is 605\n",
      "Normal_start:1\n",
      "49037\n",
      "0.616275766704\n",
      "episode is 606\n",
      "Random_start:50\n",
      "49046\n",
      "0.616221204341\n",
      "episode is 607\n",
      "Normal_start:1\n",
      "49127\n",
      "0.615730363982\n",
      "episode is 608\n",
      "Random_start:77\n",
      "49140\n",
      "0.615651624153\n",
      "episode is 609\n",
      "Random_start:26\n",
      "49156\n",
      "0.615554727645\n",
      "episode is 610\n",
      "Random_start:97\n",
      "49186\n",
      "0.615373088474\n",
      "episode is 611\n",
      "Normal_start:1\n",
      "49246\n",
      "0.615009973566\n",
      "episode is 612\n",
      "Normal_start:1\n",
      "49285\n",
      "0.614774065682\n",
      "episode is 613\n",
      "Random_start:28\n",
      "49302\n",
      "0.614671262829\n",
      "episode is 614\n",
      "Random_start:96\n",
      "49310\n",
      "0.614622891063\n",
      "episode is 615\n",
      "Random_start:98\n",
      "49312\n",
      "0.614610798726\n",
      "episode is 616\n",
      "Random_start:9\n",
      "49327\n",
      "0.614520113908\n",
      "episode is 617\n",
      "Random_start:8\n",
      "49340\n",
      "0.614441531401\n",
      "episode is 618\n",
      "Random_start:72\n",
      "49366\n",
      "0.614284397031\n",
      "episode is 619\n",
      "Normal_start:1\n",
      "49417\n",
      "0.613976290562\n",
      "episode is 620\n",
      "Normal_start:1\n",
      "49466\n",
      "0.613680414676\n",
      "episode is 621\n",
      "Normal_start:1\n",
      "49501\n",
      "0.613469163502\n",
      "episode is 622\n",
      "Random_start:96\n",
      "49525\n",
      "0.613324348281\n",
      "episode is 623\n",
      "Normal_start:1\n",
      "49584\n",
      "0.612968491903\n",
      "episode is 624\n",
      "Normal_start:1\n",
      "49631\n",
      "0.6126851633\n",
      "episode is 625\n",
      "Random_start:79\n",
      "49638\n",
      "0.612642976815\n",
      "episode is 626\n",
      "Normal_start:1\n",
      "49725\n",
      "0.612118905429\n",
      "episode is 627\n",
      "Random_start:79\n",
      "49751\n",
      "0.611962374863\n",
      "episode is 628\n",
      "Normal_start:1\n",
      "49819\n",
      "0.611553179591\n",
      "episode is 629\n",
      "Random_start:39\n",
      "49847\n",
      "0.611384768279\n",
      "episode is 630\n",
      "Normal_start:1\n",
      "49899\n",
      "0.611072129493\n",
      "episode is 631\n",
      "Normal_start:1\n",
      "49952\n",
      "0.61075364567\n",
      "episode is 632\n",
      "Random_start:73\n",
      "49986\n",
      "0.61054942415\n",
      "episode is 633\n",
      "Normal_start:1\n",
      "50022\n",
      "0.610333265268\n",
      "episode is 634\n",
      "Normal_start:1\n",
      "50057\n",
      "0.610123185391\n",
      "episode is 635\n",
      "Random_start:78\n",
      "50067\n",
      "0.610063176073\n",
      "episode is 636\n",
      "Random_start:8\n",
      "50095\n",
      "0.609895181904\n",
      "episode is 637\n",
      "Normal_start:1\n",
      "50151\n",
      "0.609559334648\n",
      "episode is 638\n",
      "Random_start:33\n",
      "50206\n",
      "0.609229667681\n",
      "episode is 639\n",
      "Random_start:77\n",
      "50213\n",
      "0.609187723072\n",
      "episode is 640\n",
      "Normal_start:1\n",
      "50286\n",
      "0.608750475649\n",
      "episode is 641\n",
      "Random_start:89\n",
      "50292\n",
      "0.608714551698\n",
      "episode is 642\n",
      "Random_start:48\n",
      "50323\n",
      "0.608528978953\n",
      "episode is 643\n",
      "Random_start:18\n",
      "50339\n",
      "0.608433221977\n",
      "episode is 644\n",
      "Random_start:24\n",
      "50361\n",
      "0.608301581149\n",
      "episode is 645\n",
      "Random_start:78\n",
      "50367\n",
      "0.608265684131\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode is 646\n",
      "Random_start:92\n",
      "50434\n",
      "0.607864980373\n",
      "episode is 647\n",
      "Normal_start:1\n",
      "50453\n",
      "0.607751396818\n",
      "episode is 648\n",
      "Random_start:64\n",
      "50496\n",
      "0.607494418972\n",
      "episode is 649\n",
      "Normal_start:1\n",
      "50529\n",
      "0.607297278343\n",
      "episode is 650\n",
      "Random_start:72\n",
      "50595\n",
      "0.606903192202\n",
      "episode is 651\n",
      "Normal_start:1\n",
      "50641\n",
      "0.606628679877\n",
      "episode is 652\n",
      "Random_start:41\n",
      "50672\n",
      "0.606443753651\n",
      "episode is 653\n",
      "Random_start:37\n",
      "50701\n",
      "0.60627081004\n",
      "episode is 654\n",
      "Random_start:37\n",
      "50722\n",
      "0.606145606317\n",
      "episode is 655\n",
      "Random_start:36\n",
      "50756\n",
      "0.605942951264\n",
      "episode is 656\n",
      "Random_start:81\n",
      "50802\n",
      "0.605668880548\n",
      "episode is 657\n",
      "Normal_start:1\n",
      "50827\n",
      "0.605519981941\n",
      "episode is 658\n",
      "Random_start:97\n",
      "50836\n",
      "0.605466387554\n",
      "episode is 659\n",
      "Random_start:18\n",
      "50897\n",
      "0.605103263822\n",
      "episode is 660\n",
      "Random_start:83\n",
      "50935\n",
      "0.604877167542\n",
      "episode is 661\n",
      "Random_start:35\n",
      "50962\n",
      "0.604716572389\n",
      "episode is 662\n",
      "Normal_start:1\n",
      "51009\n",
      "0.604437121276\n",
      "episode is 663\n",
      "Random_start:39\n",
      "51020\n",
      "0.604371736789\n",
      "episode is 664\n",
      "Random_start:81\n",
      "51059\n",
      "0.604139977007\n",
      "episode is 665\n",
      "Random_start:94\n",
      "51145\n",
      "0.603629236277\n",
      "episode is 666\n",
      "Normal_start:1\n",
      "51181\n",
      "0.603415568215\n",
      "episode is 667\n",
      "Normal_start:1\n",
      "51214\n",
      "0.603219773385\n",
      "episode is 668\n",
      "Random_start:31\n",
      "51236\n",
      "0.60308927939\n",
      "episode is 669\n",
      "Normal_start:1\n",
      "51298\n",
      "0.602721678005\n",
      "episode is 670\n",
      "Normal_start:1\n",
      "51334\n",
      "0.602508336604\n",
      "episode is 671\n",
      "Random_start:65\n",
      "51363\n",
      "0.602336534099\n",
      "episode is 672\n",
      "Random_start:66\n",
      "51388\n",
      "0.602188468475\n",
      "episode is 673\n",
      "Normal_start:1\n",
      "51445\n",
      "0.60185101723\n",
      "episode is 674\n",
      "Normal_start:1\n",
      "51512\n",
      "0.60145460986\n",
      "episode is 675\n",
      "Random_start:30\n",
      "51532\n",
      "0.601336330767\n",
      "episode is 676\n",
      "Random_start:99\n",
      "51534\n",
      "0.601324504158\n",
      "episode is 677\n",
      "Normal_start:1\n",
      "51581\n",
      "0.601046646943\n",
      "episode is 678\n",
      "Normal_start:1\n",
      "51643\n",
      "0.600680311597\n",
      "episode is 679\n",
      "Random_start:82\n",
      "51698\n",
      "0.60035552675\n",
      "episode is 680\n",
      "Random_start:15\n",
      "51717\n",
      "0.600243369855\n",
      "episode is 681\n",
      "Normal_start:1\n",
      "51773\n",
      "0.599912926101\n",
      "episode is 682\n",
      "Random_start:71\n",
      "51816\n",
      "0.599659318072\n",
      "episode is 683\n",
      "Normal_start:1\n",
      "51882\n",
      "0.599270271322\n",
      "episode is 684\n",
      "Normal_start:1\n",
      "51972\n",
      "0.598740166661\n",
      "episode is 685\n",
      "Random_start:22\n",
      "52004\n",
      "0.598551799948\n",
      "episode is 686\n",
      "Normal_start:1\n",
      "52043\n",
      "0.598322309499\n",
      "episode is 687\n",
      "Random_start:10\n",
      "52054\n",
      "0.598257597604\n",
      "episode is 688\n",
      "Random_start:28\n",
      "52080\n",
      "0.59810467051\n",
      "episode is 689\n",
      "Random_start:97\n",
      "52084\n",
      "0.598081146794\n",
      "episode is 690\n",
      "Random_start:47\n",
      "52106\n",
      "0.597951783172\n",
      "episode is 691\n",
      "Random_start:49\n",
      "52133\n",
      "0.59779305762\n",
      "episode is 692\n",
      "Random_start:68\n",
      "52200\n",
      "0.597399368172\n",
      "episode is 693\n",
      "Normal_start:1\n",
      "52227\n",
      "0.597240791751\n",
      "episode is 694\n",
      "Random_start:67\n",
      "52235\n",
      "0.597193814367\n",
      "episode is 695\n",
      "Random_start:92\n",
      "52263\n",
      "0.597029423115\n",
      "episode is 696\n",
      "Random_start:95\n",
      "52347\n",
      "0.596536525445\n",
      "episode is 697\n",
      "Random_start:23\n",
      "52370\n",
      "0.596401637557\n",
      "episode is 698\n",
      "Normal_start:1\n",
      "52435\n",
      "0.596020600343\n",
      "episode is 699\n",
      "Random_start:8\n",
      "52454\n",
      "0.595909267006\n",
      "11\n",
      "0.5 0.5\n",
      "0.5 1.5\n",
      "12\n",
      "0.5 1.5\n",
      "1.5 1.5\n",
      "22\n",
      "1.5 1.5\n",
      "1.5 2.5\n",
      "32\n",
      "1.5 2.5\n",
      "1.5 3.5\n",
      "42\n",
      "1.5 3.5\n",
      "1.5 4.5\n",
      "52\n",
      "1.5 4.5\n",
      "1.5 5.5\n",
      "62\n",
      "1.5 5.5\n",
      "1.5 6.5\n",
      "72\n",
      "1.5 6.5\n",
      "1.5 7.5\n",
      "73\n",
      "1.5 7.5\n",
      "2.5 7.5\n",
      "74\n",
      "2.5 7.5\n",
      "3.5 7.5\n",
      "64\n",
      "3.5 7.5\n",
      "3.5 6.5\n",
      "54\n",
      "3.5 6.5\n",
      "3.5 5.5\n",
      "44\n",
      "3.5 5.5\n",
      "3.5 4.5\n",
      "45\n",
      "3.5 4.5\n",
      "4.5 4.5\n",
      "46\n",
      "4.5 4.5\n",
      "5.5 4.5\n",
      "47\n",
      "5.5 4.5\n",
      "6.5 4.5\n",
      "48\n",
      "6.5 4.5\n",
      "7.5 4.5\n",
      "49\n",
      "7.5 4.5\n",
      "8.5 4.5\n",
      "59\n",
      "8.5 4.5\n",
      "8.5 5.5\n",
      "69\n",
      "8.5 5.5\n",
      "8.5 6.5\n",
      "70\n",
      "8.5 6.5\n",
      "9.5 6.5\n",
      "80\n",
      "9.5 6.5\n",
      "9.5 7.5\n",
      "90\n",
      "9.5 7.5\n",
      "9.5 8.5\n",
      "100\n",
      "9.5 8.5\n",
      "9.5 9.5\n"
     ]
    }
   ],
   "source": [
    "%matplotlib qt5\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "Alpha=0.1 #学习速率\n",
    "Beta=0.9 #对未来奖励的衰减度\n",
    "Epsilon_start=1#选择随机动作的概率 Epsilon\n",
    "Epsilon_stop=0.01\n",
    "decay_rate=0.00001\n",
    "N_Mesh=10\n",
    "N_States=100#假设网格为5*5\n",
    "Actions=['left','right','up','down']\n",
    "Obstacle=[3,5,13,16,21,23,27,29,31,33,34,43,53,56,57,63,66,67,76,78,81,86,96,98]\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Caculate_coordinate(State):\n",
    "    X_Axis=0.5+(State-1)%N_Mesh\n",
    "    Y_Axis=0.5+((State-1)//N_Mesh)\n",
    "    return X_Axis,Y_Axis\n",
    "def Initial_Q_Table (N_States,Actions):#以状态数和动作为变量，返回Q表\n",
    "    Q_Table=np.zeros([N_States,len(Actions)])#初始化Q_表\n",
    "    Q_Table=pd.DataFrame(Q_Table,index=np.arange(1,N_States+1),columns=Actions)#Q_表定义行列标签\n",
    "    print ('初始化Q表完成') \n",
    "    return Q_Table\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Select_Action(State,Q_Table,ep):#根据目前的状态选择动作,返回执行动作\n",
    "    Epsilon=Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-decay_rate*ep)\n",
    "    States_Actions=Q_Table.loc[[State],:]#选择目前状态下，取出Q表的目前的状态行,state选择为数字\n",
    "    if(np.random.rand()<Epsilon or np.all(States_Actions==[0])):#其实np.all(States_Action==[0])也行\n",
    "        Execute_Action=np.random.choice(Actions)\n",
    "    else:\n",
    "        #Q_T=Q_Table.T\n",
    "        #G=(Q_Table.T).loc[:,[State]]\n",
    "        Execute_Action=np.array(pd.DataFrame.idxmax((Q_Table.T).loc[:,[State]]))[0]#这里真心累，有毒\n",
    "    return Execute_Action\n",
    "#———————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Next_State(State,Execute_Action):#根据目前状态个选择的动作,返回即时奖励和新状态\n",
    "    if(Execute_Action=='left'):\n",
    "        #目标在右上角，如果Execute_Action为‘left’，不可能到达目标\n",
    "        if(State%N_Mesh==1):#判断是不是在网格最左边\n",
    "            State=State\n",
    "            R=-10000\n",
    "        elif((State+1) in Obstacle):\n",
    "            State=State-1\n",
    "            R=-10000\n",
    "        else:\n",
    "            State=State-1\n",
    "            R=0\n",
    "    elif(Execute_Action=='right'):\n",
    "        if(State==N_States-1):\n",
    "            State='End'\n",
    "            R=10\n",
    "        elif(State%N_Mesh==0):\n",
    "            State=State\n",
    "            R=-10000\n",
    "        elif((State+1) in Obstacle):\n",
    "            State=State+1\n",
    "            R=-10000\n",
    "        else:\n",
    "            State=State+1\n",
    "            R=0.002\n",
    "    elif(Execute_Action=='up'):\n",
    "        if(State==N_States-N_Mesh):\n",
    "            State='End'\n",
    "            R=10\n",
    "        elif(N_States+1-N_Mesh<=State<=N_States-1):\n",
    "            State=State\n",
    "            R=-10000\n",
    "        elif((State+10) in Obstacle):\n",
    "            State=State+10\n",
    "            R=-10000\n",
    "        else:\n",
    "            State=State+N_Mesh\n",
    "            R=0.002\n",
    "    else:\n",
    "        if(0<=State<=N_Mesh):\n",
    "            State=State\n",
    "            R=-10000\n",
    "        elif((State-10) in Obstacle):\n",
    "            State=State-10\n",
    "            R=-10000\n",
    "        else:\n",
    "            R=0\n",
    "            State=State-10\n",
    "    return State,R\n",
    "        \n",
    "#————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Run_Function():\n",
    "    #——————————————————————————————————————————————————画图程序部分\n",
    "    \n",
    "    \n",
    "    #——————————————————————————————————————————————————\n",
    "    Q_Table=Initial_Q_Table (N_States,Actions)\n",
    "    Step=0 \n",
    "    for episode in np.arange(700) :#跑20回合\n",
    "        if(episode%300==0):\n",
    "            print('Q_Table,%d'%episode)\n",
    "            Q_Table\n",
    "#         fig=plt.figure()\n",
    "#         ax=fig.gca()\n",
    "#         ax.set(xlim=[0, N_Mesh], ylim=[0, N_Mesh])\n",
    "#         ax.set_xticks(np.arange(0,(N_Mesh+1)))\n",
    "#         ax.set_yticks(np.arange(0,(N_Mesh+1)))\n",
    "#         plt.grid()\n",
    "        print('episode is %d' % episode)\n",
    "        if(np.random.rand()<0.4):\n",
    "            State_=1 #定义起点 \n",
    "            print(\"Normal_start:%d\"%State_)\n",
    "        else:\n",
    "            State_=np.random.randint(low=5, high=100)\n",
    "            print(\"Random_start:%d\"%State_)\n",
    "        Is_End=False\n",
    "        while not Is_End:\n",
    "            A=Select_Action(State_,Q_Table,Step)\n",
    "            Next_S,R=Next_State(State_,A)\n",
    "            #print(Next_S)\n",
    "            Q_Table_S_A=Q_Table.loc[[State_],[A]]\n",
    "            if Next_S!='End':\n",
    "                D_T=np.array((Q_Table.loc[[Next_S],:]).T)\n",
    "                Q_Target=R+Beta*D_T.max() #Next_S状态中的可选动作中的最大Q值动作\n",
    "                Step+=1\n",
    "            else:\n",
    "                Q_Target=R\n",
    "                Step+=1\n",
    "                Is_End=True #目的是结束循\n",
    "#————————————————————————————————————————————————————————————————————————————画图程序\n",
    "#             X_State,Y_State=Caculate_coordinate(State_)\n",
    "#             if(Next_S)=='End':\n",
    "#                 H=N_States\n",
    "#             else:\n",
    "#                 H=Next_S\n",
    "#             X_Next_State,Y_Next_State=Caculate_coordinate(H)                      \n",
    "#             if(Step==0):\n",
    "#                 plt.scatter(X_State,Y_State)  \n",
    "#             else:\n",
    "#                 plt.scatter(X_State,Y_State) \n",
    "#                 plt.scatter(X_Next_State,Y_Next_State)\n",
    "#                 plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "#————————————————————————————————————————————————————————————————————————————\n",
    "            Q_Table.loc[[State_],[A]]+=Alpha*(Q_Target-Q_Table_S_A)\n",
    "            State_=Next_S\n",
    "        print(Step)\n",
    "        print(Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-decay_rate*Step))\n",
    "        #plt.show()\n",
    "        #print(Q_Table)\n",
    "    return Q_Table\n",
    "#————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "Q__Table=Run_Function()\n",
    "Q__Table\n",
    "Final_Q_Table=np.array(Q__Table)\n",
    "#—————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Plot_Final_Graph(Q_TABLE):\n",
    "    fig=plt.figure()\n",
    "    ax=fig.gca()\n",
    "    ax.set(xlim=[0, N_Mesh], ylim=[0, N_Mesh])\n",
    "    ax.set_xticks(np.arange(0,(N_Mesh+1)))\n",
    "    ax.set_yticks(np.arange(0,(N_Mesh+1)))\n",
    "    plt.grid()\n",
    "    State_Start=1\n",
    "    State=State_Start\n",
    "    Step=1\n",
    "    while(State!=N_States):\n",
    "        Action=np.argmax((Q_TABLE[State-1]))\n",
    "        if(Action==0):\n",
    "             Next_State=State-1\n",
    "        elif(Action==1):\n",
    "             Next_State=State+1\n",
    "        elif(Action==2):\n",
    "             Next_State=State+N_Mesh\n",
    "        else:\n",
    "             Next_State=State-N_Mesh\n",
    "        Step+=1\n",
    "        print(Next_State)\n",
    "        X_State,Y_State=Caculate_coordinate(State)\n",
    "        X_Next_State,Y_Next_State=Caculate_coordinate(Next_State)\n",
    "        if(Step==0):\n",
    "            plt.scatter(X_State,Y_State)  \n",
    "        else:\n",
    "            plt.scatter(X_State,Y_State) \n",
    "            plt.scatter(X_Next_State,Y_Next_State)\n",
    "            plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "        for J in np.arange(len(Obstacle)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle[J]))[0],(Caculate_coordinate(Obstacle[J]))[1],marker=\"x\")\n",
    "        print(X_State,Y_State)\n",
    "        print(X_Next_State,Y_Next_State)\n",
    "        State=Next_State\n",
    "    plt.show()\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "Plot_Final_Graph(Final_Q_Table)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
